{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bda627ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/Cellar/apache-spark/3.2.1/libexec/conf/spark-env.sh: line 1: spark.executor.memory=6g: command not found\n",
      "2022-04-21 21:14:05,804 WARN util.Utils: Your hostname, huzujundeMacBook-Pro.local resolves to a loopback address: 127.0.0.1; using 192.168.3.6 instead (on interface en0)\n",
      "2022-04-21 21:14:05,804 WARN util.Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
      "WARNING: An illegal reflective access operation has occurred\n",
      "WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/usr/local/Cellar/apache-spark/3.2.1/libexec/jars/spark-unsafe_2.12-3.2.1.jar) to constructor java.nio.DirectByteBuffer(long,int)\n",
      "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n",
      "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n",
      "WARNING: All illegal access operations will be denied in a future release\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "2022-04-21 21:14:06,371 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "2022-04-21 21:14:07,284 WARN util.Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n"
     ]
    }
   ],
   "source": [
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import col, locate, regexp_extract, array_position, udf, expr, when\n",
    "from pyspark.sql.types import StructType, StructField, StringType, ArrayType\n",
    "from pypinyin import lazy_pinyin, Style\n",
    "\n",
    "from typing import List, Tuple\n",
    "#from phase import word_parser\n",
    "import time\n",
    "import re\n",
    "\n",
    "\n",
    "sc = pyspark.SparkContext(appName=\"demo\")\n",
    "spark = SparkSession(sc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2bb73e98",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def word_parser(sentence: str) -> Tuple[List[str], List[str]]:\n",
    "    pinyins = lazy_pinyin(sentence)\n",
    "    rhys = pinyin_parser(pinyins)\n",
    "    rhys_tone = add_tone(sentence, rhys)\n",
    "    return rhys, rhys_tone\n",
    "\n",
    "def add_tone(sentence, ryhs):\n",
    "    pinyins_tone = lazy_pinyin(sentence, style=Style.FINALS_TONE3)\n",
    "    result = []\n",
    "    for i, ryh in enumerate(ryhs):\n",
    "        if pinyins_tone[i][-1] == '1' or pinyins_tone[i][-1] == '2' or pinyins_tone[i][-1] == '3' or pinyins_tone[i][-1] == '4':\n",
    "            result.append(ryh+pinyins_tone[i][-1])\n",
    "        else:\n",
    "            result.append(ryh)\n",
    "    return result\n",
    "\n",
    "\n",
    "def pinyin_parser(pinyins: List[str]) -> List[List[str]]:\n",
    "    parsed_pinyins = []\n",
    "    for pinyin in pinyins:\n",
    "        pinyin = pinyin_correction(pinyin)\n",
    "        consonant, vowel = split_cv(pinyin)\n",
    "        parsed_pinyins.append(weak_rhyme(vowel_parser(vowel)))\n",
    "    return parsed_pinyins\n",
    "\n",
    "\n",
    "def pinyin_correction(pinyin: str) -> str:\n",
    "\n",
    "    # z/c/s + i -> z/c/s + I\n",
    "    if re.match(r'[zcs]i$', pinyin):\n",
    "        return pinyin.replace('i', 'I')\n",
    "\n",
    "    # zh/ch/sh/r + i -> zh/ch/sh/r + II\n",
    "    elif re.match(r'(?:zh|ch|sh|r)i$', pinyin):\n",
    "        return pinyin.replace('i', 'II')\n",
    "\n",
    "    # j/q/x/y + u/ue/un/uan -> j/q/x + v/ve/vn/van | v/ve/vn/van\n",
    "    # yu == yi\n",
    "    elif re.match(r'[jqxy]u', pinyin):\n",
    "        if pinyin.endswith('u'):\n",
    "            return re.sub(r'y*(.+)', r'\\1', pinyin.replace('u', 'i'))\n",
    "        return re.sub(r'y*(.+)', r'\\1', pinyin.replace('u', 'v'))\n",
    "\n",
    "    # y + a/e/ao/ou/an/ in/iang/ing/iong -> ia/ie/iao/iou/ian/ in/iang/ing/iong\n",
    "    elif pinyin.startswith(\"y\"):\n",
    "        return re.sub(r'yi*(.*)', r'i\\1', pinyin)\n",
    "\n",
    "    # w + u/a/o/ai/ei/an/en/ang/eng        -> u/ua/uo/uai/uei/uan/uen/uang/ueng\n",
    "    elif pinyin.startswith(\"w\"):\n",
    "        return re.sub(r'wu*(.*)', r'u\\1', pinyin)\n",
    "\n",
    "    # qiu -> qiou\n",
    "    elif pinyin.endswith('iu'):\n",
    "        return pinyin.replace('iu', 'iou')\n",
    "\n",
    "    # cui -> cuei\n",
    "    elif pinyin.endswith('ui'):\n",
    "        return pinyin.replace('ui', 'uei')\n",
    "\n",
    "    # lun -> luen\n",
    "    elif pinyin.endswith('un'):\n",
    "        return pinyin.replace('un', 'uen')\n",
    "\n",
    "\n",
    "    return pinyin\n",
    "\n",
    "\n",
    "def split_cv(pinyin):\n",
    "    return re.findall(r'(ch|zh|sh|[^aeiIouv])*(.+)', pinyin)[0]\n",
    "\n",
    "\n",
    "def vowel_parser(vowel: str) -> str:\n",
    "    if vowel == 'van':\n",
    "        return 'an'\n",
    "\n",
    "    elif len(vowel) > 1 and vowel[0] == 'u':\n",
    "        return vowel[1:]\n",
    "\n",
    "    # for 'i', except 'in' and 'ing' (one vowel), 'ie' and 'ian' (sound different from 'e' and 'an')\n",
    "    elif len(vowel) > 1 and vowel[0] == 'i' and vowel[1] != 'n' and vowel[1] != 'e' and vowel[1:] != 'an':\n",
    "        return vowel[1:]\n",
    "\n",
    "    return vowel\n",
    "\n",
    "def weak_rhyme(pinyin):\n",
    "    if pinyin.endswith('eng'):\n",
    "        return pinyin.replace('eng', 'en')\n",
    "\n",
    "    elif pinyin.endswith('ang'):\n",
    "        return pinyin.replace('ang', 'an')\n",
    "\n",
    "    elif pinyin.endswith('ing'):\n",
    "        return pinyin.replace('ing', 'in')\n",
    "\n",
    "    elif pinyin.endswith('ui'):\n",
    "        return pinyin.replace('ui', 'ei')\n",
    "\n",
    "    elif pinyin.endswith('ie'):\n",
    "        return pinyin.replace('ui', 'ei')\n",
    "\n",
    "    elif pinyin.endswith('ue'):\n",
    "        return pinyin.replace('ue', 'ei')\n",
    "\n",
    "    return pinyin\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a218dee2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------------------------------+--------------------+-------------------------+--------------------+----------------------+\n",
      "|                               sentences| pinyin_without_tone|pinyin_without_tone_split|    pinyin_with_tone|pinyin_with_tone_split|\n",
      "+----------------------------------------+--------------------+-------------------------+--------------------+----------------------+\n",
      "|                                签约了他|'ian', 've', 'e',...|              [5, 11, 16]|'ian1', 've1', 'e...|           [6, 13, 18]|\n",
      "|   點稱呼隨便你歐陽靖定無野叻最出色系...|'ian', 'en', 'u',...|     [5, 11, 16, 22, 2...|'ian3', 'en1', 'u...|  [6, 13, 19, 26, 3...|\n",
      "|                          要开心就要喝酒|'ao', 'ai', 'in',...|     [4, 10, 16, 22, 2...|'ao4', 'ai1', 'in...|  [5, 12, 19, 26, 3...|\n",
      "|                      我虽然喺美国出世啊|'o', 'ei', 'an', ...|     [3, 9, 15, 20, 26...|'o3', 'ei1', 'an2...|  [4, 11, 18, 24, 3...|\n",
      "|                            所以别总说你|'o', 'i', 'ie', '...|       [3, 8, 14, 21, 26]|'o3', 'i3', 'ie2'...|   [4, 10, 17, 25, 31]|\n",
      "|                        野心贯穿南方北方|'ie', 'in', 'an',...|     [4, 10, 16, 22, 2...|'ie3', 'in1', 'an...|  [5, 12, 19, 26, 3...|\n",
      "|        眼睛有障碍看东西都像没戴眼镜看裸|'ian', 'in', 'ou'...|     [5, 11, 17, 23, 2...|'ian3', 'in1', 'o...|  [6, 13, 20, 27, 3...|\n",
      "|                                香港嘅啊|'an', 'an', 'ai',...|              [4, 10, 16]|'an1', 'an3', 'ai...|           [5, 12, 19]|\n",
      "|                            比以前更强了|'i', 'i', 'ian', ...|       [3, 8, 15, 21, 27]|'i3', 'i3', 'ian2...|   [4, 10, 18, 25, 32]|\n",
      "|缠绕让你乱套把你干掉看好给你制造一些烦恼|'an', 'ao', 'an',...|     [4, 10, 16, 21, 2...|'an2', 'ao4', 'an...|  [5, 12, 19, 25, 3...|\n",
      "|                                    大夫|           'ai', 'u'|                      [4]|         'ai4', 'u1'|                   [5]|\n",
      "|              不会吧只会是想蹿想逃串想搬|'u', 'ei', 'a', '...|     [3, 9, 14, 20, 26...|'u2', 'ei4', 'a',...|  [4, 11, 16, 23, 3...|\n",
      "|          告诉我有什么用如果我只知道法则|'ao', 'u', 'o', '...|     [4, 9, 14, 20, 26...|'ao4', 'u4', 'o3'...|  [5, 11, 17, 24, 3...|\n",
      "|  我想要触天空所以我正在伸手去触摸天花板|'o', 'an', 'ao', ...|     [3, 9, 15, 20, 27...|'o3', 'an3', 'ao4...|  [4, 11, 18, 24, 3...|\n",
      "|                    不知不觉时间五年已过|'u', 'II', 'u', '...|     [3, 9, 14, 20, 26...|'u4', 'II1', 'u4'...|  [4, 11, 17, 24, 3...|\n",
      "|                  我还骗你对不起我怕挨揍|'o', 'ai', 'ian',...|     [3, 9, 16, 21, 27...|'o3', 'ai2', 'ian...|  [4, 11, 19, 25, 3...|\n",
      "|                          谁都想要把自己|'ei', 'ou', 'an',...|     [4, 10, 16, 22, 2...|'ei2', 'ou1', 'an...|  [5, 12, 19, 26, 3...|\n",
      "|                          编曲文一丁荆极|'ian', 'i', 'en',...|     [5, 10, 16, 21, 2...|'ian1', 'i1', 'en...|  [6, 12, 19, 25, 3...|\n",
      "|                    供词是中文他这样陈词|'ong', 'I', 'II',...|     [5, 10, 16, 23, 2...|'ong4', 'I2', 'II...|  [6, 12, 19, 27, 3...|\n",
      "|          是学校的赘肉试卷上躺了我的遗骸|'II', 've', 'ao',...|     [4, 10, 16, 21, 2...|'II4', 've2', 'ao...|  [5, 12, 19, 24, 3...|\n",
      "+----------------------------------------+--------------------+-------------------------+--------------------+----------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = spark.read.format(\"orc\").load(\"rhyme/sentences.orc\").repartition(3).cache()\n",
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1e47bca8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.61 ms, sys: 2.41 ms, total: 8.02 ms\n",
      "Wall time: 626 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DataFrame[summary: string, rhyme: string, count: string]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def find_rhyme_without_tone(word):\n",
    "    word_pinyin = str(word_parser(word)[0])[1:-1]\n",
    "    filtered = df.filter(df.pinyin_without_tone.contains(word_pinyin))\n",
    "    sentences = filtered.select(df.sentences, locate(word_pinyin, df.pinyin_without_tone).alias('position'),\n",
    "                    df.pinyin_without_tone_split)\n",
    "    \n",
    "    sentences = sentences.withColumn('find_result', expr(\"IF(position==1, 0, array_position(pinyin_without_tone_split, position - 3) + 1)\"))\n",
    "    sentences = sentences.withColumn('rhyme', expr(f\"substring(sentences, find_result, {len(word)})\"))\n",
    "    #start = time.time() \n",
    "    sentences = sentences.groupBy(\"rhyme\").count().sort('count', ascending=False)\n",
    "    #sentences.show()\n",
    "    return sentences.describe()\n",
    "    #end = time.time()\n",
    "    #print(end - start)\n",
    "word = \"想要\"\n",
    "%time rhymes = find_rhyme_without_tone(word)\n",
    "rhymes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e4e33ca7",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'bert_serving'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_19381/3912930469.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mbert_serving\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclient\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBertClient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mbc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBertClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'bert_serving'"
     ]
    }
   ],
   "source": [
    "from bert_serving.client import BertClient\n",
    "bc = BertClient()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "310ff057",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'bc' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/n4/g1xtbtsn5gz6k5yv4ss93hfc0000gn/T/ipykernel_81501/4158322929.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mcosine_similarity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeature_vec_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeature_vec_2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mencodings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mrhymes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrhyme\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrhymes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mword_encoding\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0msimilarities\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mget_cosine_similarity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword_encoding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mencodings\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'bc' is not defined"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "def get_cosine_similarity(feature_vec_1, feature_vec_2):\n",
    "    return cosine_similarity(feature_vec_1.reshape(1, -1), feature_vec_2.reshape(1, -1))[0][0]\n",
    "\n",
    "encodings = bc.encode([rhymes[i].rhyme for i in range(min(100, len(rhymes)))])\n",
    "word_encoding = bc.encode([word])[0]\n",
    "similarities = [get_cosine_similarity(word_encoding, i) for i in encodings]\n",
    "similarities_dict = {}\n",
    "for i in range(len(similarities)):\n",
    "    similarities_dict[rhymes[i]] = similarities[i]\n",
    "similarities_dict = sorted(similarities_dict.items(), key=lambda item: item[1], reverse=True)\n",
    "similarities_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e8940fd",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from sparknlp.pretrained import PretrainedPipeline\n",
    "pipeline = PretrainedPipeline(\"translate_en_zh\", lang = \"xx\")\n",
    "pipeline.annotate(\"Your sentence to translate!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1729d91",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "lazy_pinyin(\"欢喜\", style=Style.FINALS_TONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b902343f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----+\n",
      "|rhyme|count|\n",
      "+-----+-----+\n",
      "| 继续|  528|\n",
      "| 记忆|  134|\n",
      "| 秘密|  116|\n",
      "| 意义|   83|\n",
      "| 地狱|   51|\n",
      "| 利益|   41|\n",
      "| 力气|   35|\n",
      "| 力去|   26|\n",
      "| 义气|   18|\n",
      "| 力地|   18|\n",
      "| 的地|   17|\n",
      "| 弟弟|   15|\n",
      "| 繼續|   15|\n",
      "| 地利|   14|\n",
      "| 係記|   12|\n",
      "| 抑郁|   11|\n",
      "| 去去|   11|\n",
      "| 意去|   10|\n",
      "| 翼翼|   10|\n",
      "| 地去|   10|\n",
      "+-----+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "[Row(rhyme='继续', count=528), Row(rhyme='记忆', count=134), Row(rhyme='秘密', count=116), Row(rhyme='意义', count=83), Row(rhyme='地狱', count=51), Row(rhyme='利益', count=41), Row(rhyme='力气', count=35), Row(rhyme='力去', count=26), Row(rhyme='义气', count=18), Row(rhyme='力地', count=18), Row(rhyme='的地', count=17), Row(rhyme='繼續', count=15), Row(rhyme='弟弟', count=15), Row(rhyme='地利', count=14), Row(rhyme='係記', count=12), Row(rhyme='抑郁', count=11), Row(rhyme='去去', count=11), Row(rhyme='意去', count=10), Row(rhyme='翼翼', count=10), Row(rhyme='地去', count=10), Row(rhyme='细腻', count=10), Row(rhyme='第一', count=9), Row(rhyme='密密', count=8), Row(rhyme='系第', count=8), Row(rhyme='意力', count=8), Row(rhyme='立地', count=8), Row(rhyme='去计', count=8), Row(rhyme='细细', count=8), Row(rhyme='续续', count=7), Row(rhyme='毅力', count=7), Row(rhyme='必去', count=7), Row(rhyme='异地', count=6), Row(rhyme='弟一', count=6), Row(rhyme='意地', count=6), Row(rhyme='系去', count=5), Row(rhyme='记第', count=5), Row(rhyme='戏一', count=5), Row(rhyme='戏剧', count=5), Row(rhyme='距地', count=5), Row(rhyme='句句', count=5), Row(rhyme='蜜蜜', count=5), Row(rhyme='觅觅', count=4), Row(rhyme='易地', count=4), Row(rhyme='地记', count=4), Row(rhyme='去一', count=4), Row(rhyme='记去', count=4), Row(rhyme='戾气', count=4), Row(rhyme='系系', count=4), Row(rhyme='地气', count=4), Row(rhyme='力继', count=3), Row(rhyme='屹立', count=3), Row(rhyme='际遇', count=3), Row(rhyme='绪一', count=3), Row(rhyme='去继', count=3), Row(rhyme='意義', count=3), Row(rhyme='寓意', count=3), Row(rhyme='利毕', count=3), Row(rhyme='气力', count=3), Row(rhyme='去记', count=3), Row(rhyme='历历', count=3), Row(rhyme='意气', count=3), Row(rhyme='气去', count=2), Row(rhyme='系一', count=2), Row(rhyme='必必', count=2), Row(rhyme='气预', count=2), Row(rhyme='粒粒', count=2), Row(rhyme='气闭', count=2), Row(rhyme='聚力', count=2), Row(rhyme='郁郁', count=2), Row(rhyme='去觅', count=2), Row(rhyme='系记', count=2), Row(rhyme='欲拒', count=2), Row(rhyme='弟去', count=2), Row(rhyme='利欲', count=2), Row(rhyme='季季', count=2), Row(rhyme='去第', count=2), Row(rhyme='立立', count=2), Row(rhyme='去睇', count=2), Row(rhyme='辟地', count=2), Row(rhyme='忆去', count=2), Row(rhyme='俱寂', count=2), Row(rhyme='戏继', count=2), Row(rhyme='系立', count=2), Row(rhyme='記憶', count=2), Row(rhyme='气继', count=2), Row(rhyme='義氣', count=2), Row(rhyme='去必', count=2), Row(rhyme='地地', count=2), Row(rhyme='地义', count=2), Row(rhyme='谊去', count=2), Row(rhyme='丽丽', count=2), Row(rhyme='帝遇', count=2), Row(rhyme='豫地', count=2), Row(rhyme='的欲', count=2), Row(rhyme='异域', count=2), Row(rhyme='句必', count=2), Row(rhyme='豫豫', count=2), Row(rhyme='蜜气', count=1), Row(rhyme='历去', count=1), Row(rhyme='聚聚', count=1), Row(rhyme='利器', count=1), Row(rhyme='例遇', count=1), Row(rhyme='细粒', count=1), Row(rhyme='细地', count=1), Row(rhyme='气一', count=1), Row(rhyme='欲具', count=1), Row(rhyme='祕密', count=1), Row(rhyme='蓄力', count=1), Row(rhyme='遇第', count=1), Row(rhyme='意愈', count=1), Row(rhyme='器去', count=1), Row(rhyme='迹必', count=1), Row(rhyme='的系', count=1), Row(rhyme='計一', count=1), Row(rhyme='历叙', count=1), Row(rhyme='意继', count=1), Row(rhyme='计地', count=1), Row(rhyme='义毕', count=1), Row(rhyme='地既', count=1), Row(rhyme='去意', count=1), Row(rhyme='弟遇', count=1), Row(rhyme='惧一', count=1), Row(rhyme='帝既', count=1), Row(rhyme='意一', count=1), Row(rhyme='弊郁', count=1), Row(rhyme='计戏', count=1), Row(rhyme='去俱', count=1), Row(rhyme='利一', count=1), Row(rhyme='距剃', count=1), Row(rhyme='必忌', count=1), Row(rhyme='弟第', count=1), Row(rhyme='气地', count=1), Row(rhyme='利弊', count=1), Row(rhyme='记纪', count=1), Row(rhyme='迹去', count=1), Row(rhyme='惧去', count=1), Row(rhyme='剧一', count=1), Row(rhyme='序毕', count=1), Row(rhyme='去地', count=1), Row(rhyme='利利', count=1), Row(rhyme='欲地', count=1), Row(rhyme='弃一', count=1), Row(rhyme='易去', count=1), Row(rhyme='据艺', count=1), Row(rhyme='利地', count=1), Row(rhyme='地砌', count=1), Row(rhyme='係第', count=1), Row(rhyme='续剧', count=1), Row(rhyme='戏去', count=1), Row(rhyme='細細', count=1), Row(rhyme='戏币', count=1), Row(rhyme='屁第', count=1), Row(rhyme='历避', count=1), Row(rhyme='句一', count=1), Row(rhyme='去叙', count=1), Row(rhyme='地闭', count=1), Row(rhyme='迹一', count=1), Row(rhyme='闭翳', count=1), Row(rhyme='譬喻', count=1), Row(rhyme='系意', count=1), Row(rhyme='系艺', count=1), Row(rhyme='气气', count=1), Row(rhyme='必记', count=1), Row(rhyme='气亦', count=1), Row(rhyme='据地', count=1), Row(rhyme='计利', count=1), Row(rhyme='絮絮', count=1), Row(rhyme='弟必', count=1), Row(rhyme='地继', count=1), Row(rhyme='技艺', count=1), Row(rhyme='遇去', count=1), Row(rhyme='奕奕', count=1), Row(rhyme='系易', count=1), Row(rhyme='睥睨', count=1), Row(rhyme='技去', count=1), Row(rhyme='意立', count=1), Row(rhyme='狱地', count=1), Row(rhyme='去纪', count=1), Row(rhyme='句地', count=1), Row(rhyme='地系', count=1), Row(rhyme='哔哔', count=1), Row(rhyme='义第', count=1), Row(rhyme='地替', count=1), Row(rhyme='去避', count=1), Row(rhyme='利去', count=1), Row(rhyme='避一', count=1), Row(rhyme='愈细', count=1), Row(rhyme='睇戏', count=1), Row(rhyme='腻腻', count=1), Row(rhyme='匿迹', count=1), Row(rhyme='立壁', count=1), Row(rhyme='地域', count=1), Row(rhyme='气必', count=1), Row(rhyme='具地', count=1), Row(rhyme='去利', count=1), Row(rhyme='毅立', count=1), Row(rhyme='记记', count=1), Row(rhyme='地遇', count=1), Row(rhyme='記地', count=1), Row(rhyme='闭气', count=1), Row(rhyme='秘技', count=1), Row(rhyme='预计', count=1)]\n"
     ]
    }
   ],
   "source": [
    "def find_rhyme_with_tone(word): \n",
    "    word_pinyin = str(word_parser(word)[1])[1:-1]\n",
    "    filtered = df.filter(df.pinyin_with_tone.contains(word_pinyin))\n",
    "    sentences = filtered.select(df.sentences, locate(word_pinyin, df.pinyin_with_tone).alias('position'),\n",
    "                    df.pinyin_with_tone_split)\n",
    "    \n",
    "    sentences = sentences.withColumn('find_result', expr(\"IF(position==1, 0, array_position(pinyin_with_tone_split, position - 3) + 1)\"))\n",
    "    sentences = sentences.withColumn('rhyme', expr(f\"substring(sentences, find_result, {len(word)})\"))\n",
    "    sentences.groupBy(\"rhyme\").count().sort('count', ascending=False).show()\n",
    "    return sentences.groupBy(\"rhyme\").count().sort('count', ascending=False).collect()\n",
    "    \n",
    "print(find_rhyme_with_tone(\"秘密\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9eb2d0d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== Physical Plan ==\n",
      "*(1) Filter (isnotnull(pinyin_without_tone#197) AND Contains(pinyin_without_tone#197, 'i', 'i'))\n",
      "+- InMemoryTableScan [sentences#196, pinyin_without_tone#197, pinyin_without_tone_split#198, pinyin_with_tone#199, pinyin_with_tone_split#200], [isnotnull(pinyin_without_tone#197), Contains(pinyin_without_tone#197, 'i', 'i')]\n",
      "      +- InMemoryRelation [sentences#196, pinyin_without_tone#197, pinyin_without_tone_split#198, pinyin_with_tone#199, pinyin_with_tone_split#200], StorageLevel(disk, memory, deserialized, 1 replicas)\n",
      "            +- Exchange RoundRobinPartitioning(3), REPARTITION_BY_NUM, [id=#27]\n",
      "               +- FileScan orc [sentences#196,pinyin_without_tone#197,pinyin_without_tone_split#198,pinyin_with_tone#199,pinyin_with_tone_split#200] Batched: false, DataFilters: [], Format: ORC, Location: InMemoryFileIndex(1 paths)[hdfs://localhost:9000/user/huzujun/rhyme/sentences.orc], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<sentences:string,pinyin_without_tone:string,pinyin_without_tone_split:array<int>,pinyin_wi...\n",
      "\n",
      "\n",
      "+-------------------------------------+--------------------+-------------------------+--------------------+----------------------+\n",
      "|                            sentences| pinyin_without_tone|pinyin_without_tone_split|    pinyin_with_tone|pinyin_with_tone_split|\n",
      "+-------------------------------------+--------------------+-------------------------+--------------------+----------------------+\n",
      "|點稱呼隨便你歐陽靖定無野叻最出色系...|'ian', 'en', 'u',...|     [5, 11, 16, 22, 2...|'ian3', 'en1', 'u...|  [6, 13, 19, 26, 3...|\n",
      "|                         比以前更强了|'i', 'i', 'ian', ...|       [3, 8, 15, 21, 27]|'i3', 'i3', 'ian2...|   [4, 10, 18, 25, 32]|\n",
      "|       我要是拿枪指着你你大概会含糊地|'o', 'ao', 'II', ...|     [3, 9, 15, 20, 26...|'o3', 'ao4', 'II4...|  [4, 11, 18, 24, 3...|\n",
      "|               只可以希望通过音乐逃出|'II', 'e', 'i', '...|     [4, 9, 14, 19, 25...|'II3', 'e3', 'i3'...|  [5, 11, 17, 23, 3...|\n",
      "|       骂语气词的全是脏话倒在老子跟前|'a', 'i', 'i', 'I...|     [3, 8, 13, 18, 23...|'a4', 'i3', 'i4',...|  [4, 10, 16, 22, 2...|\n",
      "|               某一集是为了看你的欧巴|'ou', 'i', 'i', '...|     [4, 9, 14, 20, 26...|'ou3', 'i1', 'i2'...|  [5, 11, 17, 24, 3...|\n",
      "|         我有你你有你你饮香片我饮普洱|'o', 'ou', 'i', '...|     [3, 9, 14, 19, 25...|'o3', 'ou3', 'i3'...|  [4, 11, 17, 23, 3...|\n",
      "|       土豪现金拿给你穷富在一起快去比|'u', 'ao', 'ian',...|     [3, 9, 16, 22, 27...|'u3', 'ao2', 'ian...|  [4, 11, 19, 26, 3...|\n",
      "|                 你咪话下次出都谂一谂|'i', 'i', 'a', 'a...|     [3, 8, 13, 18, 23...|'i3', 'i1', 'a4',...|  [4, 10, 16, 22, 2...|\n",
      "|             热情的血脉创业是一笔血债|'e', 'in', 'e', '...|     [3, 9, 14, 20, 26...|'e4', 'in2', 'e',...|  [4, 11, 16, 23, 3...|\n",
      "|                       上帝既然给了我|'an', 'i', 'i', '...|     [4, 9, 14, 20, 26...|'an4', 'i4', 'i4'...|  [5, 11, 17, 24, 3...|\n",
      "|                   系讲点会系一盘散沙|'i', 'an', 'ian',...|     [3, 9, 16, 22, 27...|'i4', 'an3', 'ian...|  [4, 11, 19, 26, 3...|\n",
      "|             年复一年的你已成功的蜕变|'ian', 'u', 'i', ...|     [5, 10, 15, 22, 2...|'ian2', 'u4', 'i1...|  [6, 12, 18, 26, 3...|\n",
      "|               这个故事的寓意是告诉你|'e', 'e', 'u', 'I...|     [3, 8, 13, 19, 24...|'e4', 'e4', 'u4',...|  [4, 10, 16, 23, 2...|\n",
      "|         我跟我兄弟喜欢用带你们甩脑壳|'o', 'en', 'o', '...|     [3, 9, 14, 21, 26...|'o3', 'en1', 'o3'...|  [4, 11, 17, 25, 3...|\n",
      "|   问自己几年前的今天你是否还能够记起|'en', 'I', 'i', '...|     [4, 9, 14, 19, 26...|'en4', 'I4', 'i3'...|  [5, 11, 17, 23, 3...|\n",
      "| 我快要失去理智了抱着一个超大的标志牌|'o', 'ai', 'ao', ...|     [3, 9, 15, 21, 26...|'o3', 'ai4', 'ao4...|  [4, 11, 18, 25, 3...|\n",
      "|             你咪话都几发嘟吓嘟吓咁就|'i', 'i', 'a', 'o...|     [3, 8, 13, 19, 24...|'i3', 'i1', 'a4',...|  [4, 10, 16, 23, 2...|\n",
      "|           粉丝都是机器喷子还不用电的|'en', 'I', 'ou', ...|     [4, 9, 15, 21, 26...|'en3', 'I1', 'ou1...|  [5, 11, 18, 25, 3...|\n",
      "|       所以我们毫无顾忌依然握紧这个麦|'o', 'i', 'o', 'e...|     [3, 8, 13, 19, 25...|'o3', 'i3', 'o3',...|  [4, 10, 16, 22, 2...|\n",
      "+-------------------------------------+--------------------+-------------------------+--------------------+----------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "0.12352800369262695\n",
      "+-------------------------------------+--------+-------------------------+\n",
      "|                            sentences|position|pinyin_without_tone_split|\n",
      "+-------------------------------------+--------+-------------------------+\n",
      "|點稱呼隨便你歐陽靖定無野叻最出色系...|      93|     [5, 11, 16, 22, 2...|\n",
      "|                         比以前更强了|       1|       [3, 8, 15, 21, 27]|\n",
      "|       我要是拿枪指着你你大概会含糊地|      40|     [3, 9, 15, 20, 26...|\n",
      "|               只可以希望通过音乐逃出|      12|     [4, 9, 14, 19, 25...|\n",
      "|       骂语气词的全是脏话倒在老子跟前|       6|     [3, 8, 13, 18, 23...|\n",
      "|               某一集是为了看你的欧巴|       7|     [4, 9, 14, 20, 26...|\n",
      "|         我有你你有你你饮香片我饮普洱|      12|     [3, 9, 14, 19, 25...|\n",
      "|       土豪现金拿给你穷富在一起快去比|      59|     [3, 9, 16, 22, 27...|\n",
      "|                 你咪话下次出都谂一谂|       1|     [3, 8, 13, 18, 23...|\n",
      "|             热情的血脉创业是一笔血债|      47|     [3, 9, 14, 20, 26...|\n",
      "|                       上帝既然给了我|       7|     [4, 9, 14, 20, 26...|\n",
      "|                   系讲点会系一盘散沙|      25|     [3, 9, 16, 22, 27...|\n",
      "|             年复一年的你已成功的蜕变|      30|     [5, 10, 15, 22, 2...|\n",
      "|               这个故事的寓意是告诉你|      27|     [3, 8, 13, 19, 24...|\n",
      "|         我跟我兄弟喜欢用带你们甩脑壳|      24|     [3, 9, 14, 21, 26...|\n",
      "|   问自己几年前的今天你是否还能够记起|      12|     [4, 9, 14, 19, 26...|\n",
      "| 我快要失去理智了抱着一个超大的标志牌|      24|     [3, 9, 15, 21, 26...|\n",
      "|             你咪话都几发嘟吓嘟吓咁就|       1|     [3, 8, 13, 19, 24...|\n",
      "|           粉丝都是机器喷子还不用电的|      24|     [4, 9, 15, 21, 26...|\n",
      "|       所以我们毫无顾忌依然握紧这个麦|      38|     [3, 8, 13, 19, 25...|\n",
      "+-------------------------------------+--------+-------------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "0.06842207908630371\n",
      "+-------------------------------------+--------+-------------------------+-----------+\n",
      "|                            sentences|position|pinyin_without_tone_split|find_result|\n",
      "+-------------------------------------+--------+-------------------------+-----------+\n",
      "|點稱呼隨便你歐陽靖定無野叻最出色系...|      93|     [5, 11, 16, 22, 2...|         17|\n",
      "|                         比以前更强了|       1|       [3, 8, 15, 21, 27]|          0|\n",
      "|       我要是拿枪指着你你大概会含糊地|      40|     [3, 9, 15, 20, 26...|          8|\n",
      "|               只可以希望通过音乐逃出|      12|     [4, 9, 14, 19, 25...|          3|\n",
      "|       骂语气词的全是脏话倒在老子跟前|       6|     [3, 8, 13, 18, 23...|          2|\n",
      "|               某一集是为了看你的欧巴|       7|     [4, 9, 14, 20, 26...|          2|\n",
      "|         我有你你有你你饮香片我饮普洱|      12|     [3, 9, 14, 19, 25...|          3|\n",
      "|       土豪现金拿给你穷富在一起快去比|      59|     [3, 9, 16, 22, 27...|         11|\n",
      "|                 你咪话下次出都谂一谂|       1|     [3, 8, 13, 18, 23...|          0|\n",
      "|             热情的血脉创业是一笔血债|      47|     [3, 9, 14, 20, 26...|          9|\n",
      "|                       上帝既然给了我|       7|     [4, 9, 14, 20, 26...|          2|\n",
      "|                   系讲点会系一盘散沙|      25|     [3, 9, 16, 22, 27...|          5|\n",
      "|             年复一年的你已成功的蜕变|      30|     [5, 10, 15, 22, 2...|          6|\n",
      "|               这个故事的寓意是告诉你|      27|     [3, 8, 13, 19, 24...|          6|\n",
      "|         我跟我兄弟喜欢用带你们甩脑壳|      24|     [3, 9, 14, 21, 26...|          5|\n",
      "|   问自己几年前的今天你是否还能够记起|      12|     [4, 9, 14, 19, 26...|          3|\n",
      "| 我快要失去理智了抱着一个超大的标志牌|      24|     [3, 9, 15, 21, 26...|          5|\n",
      "|             你咪话都几发嘟吓嘟吓咁就|       1|     [3, 8, 13, 19, 24...|          0|\n",
      "|           粉丝都是机器喷子还不用电的|      24|     [4, 9, 15, 21, 26...|          5|\n",
      "|       所以我们毫无顾忌依然握紧这个麦|      38|     [3, 8, 13, 19, 25...|          8|\n",
      "+-------------------------------------+--------+-------------------------+-----------+\n",
      "only showing top 20 rows\n",
      "\n",
      "0.10519123077392578\n",
      "+-------------------------------------+--------+-------------------------+-----------+----------------+\n",
      "|                            sentences|position|pinyin_without_tone_split|find_result|           rhyme|\n",
      "+-------------------------------------+--------+-------------------------+-----------+----------------+\n",
      "|點稱呼隨便你歐陽靖定無野叻最出色系...|      93|     [5, 11, 16, 22, 2...|         17|  系去到邊就到邊|\n",
      "|                         比以前更强了|       1|       [3, 8, 15, 21, 27]|          0|            比以|\n",
      "|       我要是拿枪指着你你大概会含糊地|      40|     [3, 9, 15, 20, 26...|          8|你你大概会含糊地|\n",
      "|               只可以希望通过音乐逃出|      12|     [4, 9, 14, 19, 25...|          3|      以希望通过|\n",
      "|       骂语气词的全是脏话倒在老子跟前|       6|     [3, 8, 13, 18, 23...|          2|        语气词的|\n",
      "|               某一集是为了看你的欧巴|       7|     [4, 9, 14, 20, 26...|          2|        一集是为|\n",
      "|         我有你你有你你饮香片我饮普洱|      12|     [3, 9, 14, 19, 25...|          3|      你你有你你|\n",
      "|       土豪现金拿给你穷富在一起快去比|      59|     [3, 9, 16, 22, 27...|         11|      一起快去比|\n",
      "|                 你咪话下次出都谂一谂|       1|     [3, 8, 13, 18, 23...|          0|            你咪|\n",
      "|             热情的血脉创业是一笔血债|      47|     [3, 9, 14, 20, 26...|          9|        一笔血债|\n",
      "|                       上帝既然给了我|       7|     [4, 9, 14, 20, 26...|          2|        帝既然给|\n",
      "|                   系讲点会系一盘散沙|      25|     [3, 9, 16, 22, 27...|          5|      系一盘散沙|\n",
      "|             年复一年的你已成功的蜕变|      30|     [5, 10, 15, 22, 2...|          6|  你已成功的蜕变|\n",
      "|               这个故事的寓意是告诉你|      27|     [3, 8, 13, 19, 24...|          6|    寓意是告诉你|\n",
      "|         我跟我兄弟喜欢用带你们甩脑壳|      24|     [3, 9, 14, 21, 26...|          5|  弟喜欢用带你们|\n",
      "|   问自己几年前的今天你是否还能够记起|      12|     [4, 9, 14, 19, 26...|          3|      己几年前的|\n",
      "| 我快要失去理智了抱着一个超大的标志牌|      24|     [3, 9, 15, 21, 26...|          5|  去理智了抱着一|\n",
      "|             你咪话都几发嘟吓嘟吓咁就|       1|     [3, 8, 13, 19, 24...|          0|            你咪|\n",
      "|           粉丝都是机器喷子还不用电的|      24|     [4, 9, 15, 21, 26...|          5|  机器喷子还不用|\n",
      "|       所以我们毫无顾忌依然握紧这个麦|      38|     [3, 8, 13, 19, 25...|          8|忌依然握紧这个麦|\n",
      "+-------------------------------------+--------+-------------------------+-----------+----------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "0.10319900512695312\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----+\n",
      "|rhyme|count|\n",
      "+-----+-----+\n",
      "| 一起|  193|\n",
      "| 继续|  151|\n",
      "| 第一|  100|\n",
      "+-----+-----+\n",
      "\n",
      "0.4425628185272217\n"
     ]
    }
   ],
   "source": [
    "def find_rhyme_without_tone(word):\n",
    "    start = time.time()    \n",
    "    word_pinyin = str(lazy_pinyin(word, style=Style.FINALS))[1:-1]\n",
    "    filtered = df.filter(df.pinyin_without_tone.contains(word_pinyin))\n",
    "    filtered.explain()\n",
    "    filtered.show()\n",
    "    end = time.time()\n",
    "    print(end - start)\n",
    "    start = time.time()    \n",
    "    sentences = filtered.select(df.sentences, locate(word_pinyin, df.pinyin_without_tone).alias('position'),\n",
    "                    df.pinyin_without_tone_split)\n",
    "    sentences.show()\n",
    "    end = time.time()\n",
    "    print(end - start)\n",
    "    \n",
    "    start = time.time()    \n",
    "    #sentences = sentences.withColumn('find_result', find_index('position', 'pinyin_without_tone_split'))\n",
    "    #sentences = sentences.withColumn('find_result', expr(\"\"))\n",
    "    sentences = sentences.withColumn('find_result', expr(\"\"\"IF(position==1, 0, array_position(pinyin_without_tone_split, position - 3) + 1)\"\"\"))\n",
    "    sentences.show()\n",
    "    end = time.time()\n",
    "    print(end - start) \n",
    "                                     \n",
    "    @udf\n",
    "    def find_rhyme(find_result, s):\n",
    "        return s[int(find_result):int(find_result)+2]\n",
    "                                     \n",
    "    start = time.time()    \n",
    "    #sentences = sentences.withColumn('rhyme', find_rhyme('find_result', 'sentences'))\n",
    "    sentences = sentences.withColumn('rhyme', expr(\"\"\"substring(sentences, find_result, find_result+2)\"\"\"))\n",
    "    sentences.show()\n",
    "    end = time.time()\n",
    "    print(end - start)    \n",
    "                                     \n",
    "    start = time.time() \n",
    "    sentences.groupBy(\"rhyme\").count().filter(\"`count` >= 100\").sort('count', ascending=False).show()\n",
    "    end = time.time()\n",
    "    print(end - start)\n",
    "find_rhyme_without_tone(\"意义\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
